{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import product\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def plot_feat_importance(path_out, imp, oob, oos, method, tag=0, sim_num=0, **kwargs):\n",
    "    plt.figure(figure=(10, imp.shape[0] / 5.))\n",
    "    imp = imp.sort_values('mean', ascending=True)\n",
    "    ax = imp['mean'].plot(kin='barh', color='b', alpha=.25, xerr=imp['std'],\n",
    "                          error_kw={'error': 'r'})\n",
    "    if method == 'MDI':\n",
    "        plt.xlim([0,imp.sum(axis = 1).max()])\n",
    "        plt.axvline(1./imp.shape[0],linewidth = 1,color = 'r',linestyle = 'dotted')\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "\n",
    "\n",
    "def test_func(n_features=40, n_informative=10, n_redundant=10, n_estimators=1000,\n",
    "              n_samples=10000, n_splits=10):\n",
    "    X, cont = get_test_data(n_features, n_informative, n_redundant, n_samples)\n",
    "    config = {'min_w_leaf': [0.], 'scoring': ['accuracy'], 'method': ['MDI', 'MDA', 'SFI'],\n",
    "              'max_samples': [1.]}\n",
    "    jobs = [dict(zip(config.keys(), conf)) for conf in product(*config.values())]\n",
    "    kwargs = {'path_out': './test_func/', 'n_estimators': n_estimators,\n",
    "              'tag': 'test_func', 'n_splits': n_splits}\n",
    "    for job in jobs:\n",
    "        job['sim_num'] = job['method']+'_'+job['scoring']+'_'+'%.2f'%job['min_w_leaf']\\\n",
    "            + '_'+str(job['max_samples'])\n",
    "        print(job['sim_num'])\n",
    "        kwargs.udpate(job)\n",
    "        imp, oob, oos = feat_importance(X=X, cont=cont, **kwargs)\n",
    "        plot_feat_importance(imp=imp, oob=oob, oos=oos, **kwargs)\n",
    "        df0 = imp[['mean']] / imp['mean'].abs().sum()\n",
    "        df0['type'] = [i[0] for i in df0.index]\n",
    "        df0 = df0.groupby('type')['mean'].sum().to_dict()\n",
    "        df0.update({'oob': oob, 'oos': oos})\n",
    "        df0.update(job)\n",
    "        out.append(df0)\n",
    "    out = pd.DataFrame(out).sort_values(['method', 'scoring', 'min_w_leaf', 'max_sampels'])\n",
    "    out = out['method','scoring','min_w_leaf','max_samples','I','R','N','oob','oos']\n",
    "    out.to_csv(kwargs['path_out'] + 'stats.cvs')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_classification\n",
    "\n",
    "\n",
    "def get_test_data(n_features=40, n_informative=10, n_redundant=10, n_samples=10000):\n",
    "    X, cont = make_classification(n_samples=n_samples, n_features=n_features,\n",
    "                                  n_informative=n_informative, n_redundant=n_redundant,\n",
    "                                  random_state=0, shuffle=False)\n",
    "    time_idx = pd.DatetimeIndex(periods=n_samples, freq=pd.tseries.offsets.BDay(),\n",
    "                                end=pd.datetime.today())\n",
    "    X = pd.DataFrame(X, index=time_idx)\n",
    "    cont = pd.Series(cont, index=time_idx).to_frame('bin')\n",
    "    # Create name of columns\n",
    "    columns = ['I_' + str(i) for i in range(n_informative)]\n",
    "    columns += ['R_' + str(i) for i in range(n_redundant)]\n",
    "    columns += ['N_' + str(i) for i in range(n_features - len(columns))]\n",
    "    X.columns = columns\n",
    "    cont['w'] = 1. / cont.shape[0]\n",
    "    cont['t1'] = pd.Series(cont.index, index=cont.index)\n",
    "    return X, cont"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, cont = get_test_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>I_0</th>\n",
       "      <th>I_1</th>\n",
       "      <th>I_2</th>\n",
       "      <th>I_3</th>\n",
       "      <th>I_4</th>\n",
       "      <th>I_5</th>\n",
       "      <th>I_6</th>\n",
       "      <th>I_7</th>\n",
       "      <th>I_8</th>\n",
       "      <th>I_9</th>\n",
       "      <th>...</th>\n",
       "      <th>N_10</th>\n",
       "      <th>N_11</th>\n",
       "      <th>N_12</th>\n",
       "      <th>N_13</th>\n",
       "      <th>N_14</th>\n",
       "      <th>N_15</th>\n",
       "      <th>N_16</th>\n",
       "      <th>N_17</th>\n",
       "      <th>N_18</th>\n",
       "      <th>N_19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1980-03-11 16:39:48.239563</th>\n",
       "      <td>2.843740</td>\n",
       "      <td>0.456554</td>\n",
       "      <td>0.171107</td>\n",
       "      <td>-4.511382</td>\n",
       "      <td>0.278990</td>\n",
       "      <td>-3.474726</td>\n",
       "      <td>2.955550</td>\n",
       "      <td>2.698865</td>\n",
       "      <td>1.542440</td>\n",
       "      <td>2.198168</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.330515</td>\n",
       "      <td>-0.845502</td>\n",
       "      <td>-1.477466</td>\n",
       "      <td>1.217536</td>\n",
       "      <td>0.304644</td>\n",
       "      <td>1.557365</td>\n",
       "      <td>0.202843</td>\n",
       "      <td>0.160110</td>\n",
       "      <td>0.933805</td>\n",
       "      <td>-0.132272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-03-12 16:39:48.239563</th>\n",
       "      <td>3.561541</td>\n",
       "      <td>-1.566097</td>\n",
       "      <td>3.342813</td>\n",
       "      <td>-1.938909</td>\n",
       "      <td>2.075749</td>\n",
       "      <td>-3.486711</td>\n",
       "      <td>0.494908</td>\n",
       "      <td>0.309615</td>\n",
       "      <td>1.059439</td>\n",
       "      <td>-0.792433</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.020384</td>\n",
       "      <td>-0.751467</td>\n",
       "      <td>0.212077</td>\n",
       "      <td>0.285038</td>\n",
       "      <td>0.125461</td>\n",
       "      <td>0.203534</td>\n",
       "      <td>-0.376495</td>\n",
       "      <td>-0.938780</td>\n",
       "      <td>-0.142879</td>\n",
       "      <td>0.533263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-03-13 16:39:48.239563</th>\n",
       "      <td>7.699248</td>\n",
       "      <td>-3.030124</td>\n",
       "      <td>-0.859302</td>\n",
       "      <td>-0.033351</td>\n",
       "      <td>1.113719</td>\n",
       "      <td>-0.877844</td>\n",
       "      <td>2.344033</td>\n",
       "      <td>4.089113</td>\n",
       "      <td>2.287786</td>\n",
       "      <td>0.611413</td>\n",
       "      <td>...</td>\n",
       "      <td>0.744056</td>\n",
       "      <td>0.914181</td>\n",
       "      <td>1.586483</td>\n",
       "      <td>0.692802</td>\n",
       "      <td>-0.953431</td>\n",
       "      <td>0.679360</td>\n",
       "      <td>0.565153</td>\n",
       "      <td>0.219302</td>\n",
       "      <td>-1.110504</td>\n",
       "      <td>-1.086061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-03-14 16:39:48.239563</th>\n",
       "      <td>-0.149801</td>\n",
       "      <td>-3.182187</td>\n",
       "      <td>2.695894</td>\n",
       "      <td>1.359997</td>\n",
       "      <td>2.992416</td>\n",
       "      <td>-0.417971</td>\n",
       "      <td>-1.214058</td>\n",
       "      <td>1.268313</td>\n",
       "      <td>-3.720913</td>\n",
       "      <td>-2.580578</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.960632</td>\n",
       "      <td>-2.064914</td>\n",
       "      <td>1.258648</td>\n",
       "      <td>-1.031856</td>\n",
       "      <td>0.645146</td>\n",
       "      <td>-0.063900</td>\n",
       "      <td>0.305844</td>\n",
       "      <td>0.371489</td>\n",
       "      <td>3.218969</td>\n",
       "      <td>0.867178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-03-17 16:39:48.239563</th>\n",
       "      <td>-2.157903</td>\n",
       "      <td>0.046380</td>\n",
       "      <td>0.697217</td>\n",
       "      <td>-1.012036</td>\n",
       "      <td>1.856002</td>\n",
       "      <td>-2.311465</td>\n",
       "      <td>2.715493</td>\n",
       "      <td>0.444433</td>\n",
       "      <td>-1.921790</td>\n",
       "      <td>-2.472372</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.841121</td>\n",
       "      <td>0.081347</td>\n",
       "      <td>-2.587682</td>\n",
       "      <td>-0.416436</td>\n",
       "      <td>-1.077859</td>\n",
       "      <td>-0.428086</td>\n",
       "      <td>-0.183735</td>\n",
       "      <td>-0.434254</td>\n",
       "      <td>-2.124955</td>\n",
       "      <td>-0.709056</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 I_0       I_1       I_2       I_3       I_4  \\\n",
       "1980-03-11 16:39:48.239563  2.843740  0.456554  0.171107 -4.511382  0.278990   \n",
       "1980-03-12 16:39:48.239563  3.561541 -1.566097  3.342813 -1.938909  2.075749   \n",
       "1980-03-13 16:39:48.239563  7.699248 -3.030124 -0.859302 -0.033351  1.113719   \n",
       "1980-03-14 16:39:48.239563 -0.149801 -3.182187  2.695894  1.359997  2.992416   \n",
       "1980-03-17 16:39:48.239563 -2.157903  0.046380  0.697217 -1.012036  1.856002   \n",
       "\n",
       "                                 I_5       I_6       I_7       I_8       I_9  \\\n",
       "1980-03-11 16:39:48.239563 -3.474726  2.955550  2.698865  1.542440  2.198168   \n",
       "1980-03-12 16:39:48.239563 -3.486711  0.494908  0.309615  1.059439 -0.792433   \n",
       "1980-03-13 16:39:48.239563 -0.877844  2.344033  4.089113  2.287786  0.611413   \n",
       "1980-03-14 16:39:48.239563 -0.417971 -1.214058  1.268313 -3.720913 -2.580578   \n",
       "1980-03-17 16:39:48.239563 -2.311465  2.715493  0.444433 -1.921790 -2.472372   \n",
       "\n",
       "                              ...         N_10      N_11      N_12      N_13  \\\n",
       "1980-03-11 16:39:48.239563    ...    -0.330515 -0.845502 -1.477466  1.217536   \n",
       "1980-03-12 16:39:48.239563    ...    -0.020384 -0.751467  0.212077  0.285038   \n",
       "1980-03-13 16:39:48.239563    ...     0.744056  0.914181  1.586483  0.692802   \n",
       "1980-03-14 16:39:48.239563    ...    -1.960632 -2.064914  1.258648 -1.031856   \n",
       "1980-03-17 16:39:48.239563    ...    -0.841121  0.081347 -2.587682 -0.416436   \n",
       "\n",
       "                                N_14      N_15      N_16      N_17      N_18  \\\n",
       "1980-03-11 16:39:48.239563  0.304644  1.557365  0.202843  0.160110  0.933805   \n",
       "1980-03-12 16:39:48.239563  0.125461  0.203534 -0.376495 -0.938780 -0.142879   \n",
       "1980-03-13 16:39:48.239563 -0.953431  0.679360  0.565153  0.219302 -1.110504   \n",
       "1980-03-14 16:39:48.239563  0.645146 -0.063900  0.305844  0.371489  3.218969   \n",
       "1980-03-17 16:39:48.239563 -1.077859 -0.428086 -0.183735 -0.434254 -2.124955   \n",
       "\n",
       "                                N_19  \n",
       "1980-03-11 16:39:48.239563 -0.132272  \n",
       "1980-03-12 16:39:48.239563  0.533263  \n",
       "1980-03-13 16:39:48.239563 -1.086061  \n",
       "1980-03-14 16:39:48.239563  0.867178  \n",
       "1980-03-17 16:39:48.239563 -0.709056  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bin</th>\n",
       "      <th>w</th>\n",
       "      <th>t1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1980-03-11 16:39:48.239563</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>1980-03-11 16:39:48.239563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-03-12 16:39:48.239563</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>1980-03-12 16:39:48.239563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-03-13 16:39:48.239563</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>1980-03-13 16:39:48.239563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-03-14 16:39:48.239563</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>1980-03-14 16:39:48.239563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-03-17 16:39:48.239563</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>1980-03-17 16:39:48.239563</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            bin       w                         t1\n",
       "1980-03-11 16:39:48.239563    0  0.0001 1980-03-11 16:39:48.239563\n",
       "1980-03-12 16:39:48.239563    0  0.0001 1980-03-12 16:39:48.239563\n",
       "1980-03-13 16:39:48.239563    0  0.0001 1980-03-13 16:39:48.239563\n",
       "1980-03-14 16:39:48.239563    0  0.0001 1980-03-14 16:39:48.239563\n",
       "1980-03-17 16:39:48.239563    0  0.0001 1980-03-17 16:39:48.239563"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cont.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_e_vec(dot, var_thres):\n",
    "    e_val, e_vec = np.linalg.eigh(dot)\n",
    "    # Descending order\n",
    "    idx = e_val.argsort()[::-1]\n",
    "    e_val = e_val[idx]\n",
    "    e_vec = e_vec[:, idx]\n",
    "    # Use only positive ones\n",
    "    e_val = pd.Series(e_val, index=['PC_' + str(i + 1) for i in range(e_val.shape[0])])\n",
    "    e_vec = pd.DataFrame(e_vec, index=dot.index, columns=e_val.index)\n",
    "    e_vec = e_vec.loc[:, e_val > 0]\n",
    "    e_val = e_val.loc[e_val > 0]\n",
    "    # Reduce dimension with threashold\n",
    "    cum_var = e_val.cumsum() / e_val.sum()\n",
    "    dim = cum_var.values.searchsorted(var_thres)\n",
    "    e_val = e_val.iloc[:dim+1]\n",
    "    e_vec = e_vec.iloc[:, :dim+1]\n",
    "    return e_val, e_vec\n",
    "\n",
    "def orth_feats(dfX, var_thres=.95):\n",
    "    dfZ = dfX.sub(dfX.mean(), axis=1).div(dfX.std(), axis=1)\n",
    "    dot = pd.DataFrame(np.dot(dfZ.T, dfZ), index=dfX.columns, columns=dfX.columns)\n",
    "    e_val, e_vec = get_e_vec(dot, var_thres)\n",
    "    dfP = pd.DataFrame(np.dot(dfZ, e_vec), index=dfZ.index, columns=e_vec.columns)\n",
    "    return dfP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfP = orth_feats(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 28)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfP.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PC_1</th>\n",
       "      <th>PC_2</th>\n",
       "      <th>PC_3</th>\n",
       "      <th>PC_4</th>\n",
       "      <th>PC_5</th>\n",
       "      <th>PC_6</th>\n",
       "      <th>PC_7</th>\n",
       "      <th>PC_8</th>\n",
       "      <th>PC_9</th>\n",
       "      <th>PC_10</th>\n",
       "      <th>...</th>\n",
       "      <th>PC_19</th>\n",
       "      <th>PC_20</th>\n",
       "      <th>PC_21</th>\n",
       "      <th>PC_22</th>\n",
       "      <th>PC_23</th>\n",
       "      <th>PC_24</th>\n",
       "      <th>PC_25</th>\n",
       "      <th>PC_26</th>\n",
       "      <th>PC_27</th>\n",
       "      <th>PC_28</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1980-03-11 16:39:48.239563</th>\n",
       "      <td>-2.247346</td>\n",
       "      <td>3.289565</td>\n",
       "      <td>-1.367301</td>\n",
       "      <td>-2.863059</td>\n",
       "      <td>-1.412666</td>\n",
       "      <td>-0.799908</td>\n",
       "      <td>0.213012</td>\n",
       "      <td>0.331433</td>\n",
       "      <td>0.866300</td>\n",
       "      <td>0.032951</td>\n",
       "      <td>...</td>\n",
       "      <td>1.944196</td>\n",
       "      <td>-0.481867</td>\n",
       "      <td>0.060209</td>\n",
       "      <td>-0.277116</td>\n",
       "      <td>-0.534618</td>\n",
       "      <td>0.986281</td>\n",
       "      <td>-0.384909</td>\n",
       "      <td>-0.801349</td>\n",
       "      <td>0.828607</td>\n",
       "      <td>-1.121230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-03-12 16:39:48.239563</th>\n",
       "      <td>-0.956982</td>\n",
       "      <td>-0.370077</td>\n",
       "      <td>-0.070016</td>\n",
       "      <td>-3.012735</td>\n",
       "      <td>0.317126</td>\n",
       "      <td>0.332048</td>\n",
       "      <td>-0.771460</td>\n",
       "      <td>-0.385234</td>\n",
       "      <td>0.124513</td>\n",
       "      <td>0.067744</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.769808</td>\n",
       "      <td>-1.209847</td>\n",
       "      <td>-0.564018</td>\n",
       "      <td>-0.825132</td>\n",
       "      <td>0.753131</td>\n",
       "      <td>-0.379044</td>\n",
       "      <td>-0.456504</td>\n",
       "      <td>1.176804</td>\n",
       "      <td>-0.593204</td>\n",
       "      <td>0.354478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-03-13 16:39:48.239563</th>\n",
       "      <td>-4.432080</td>\n",
       "      <td>2.690585</td>\n",
       "      <td>1.475541</td>\n",
       "      <td>-1.157080</td>\n",
       "      <td>0.586217</td>\n",
       "      <td>0.561810</td>\n",
       "      <td>0.509762</td>\n",
       "      <td>-1.904502</td>\n",
       "      <td>1.081916</td>\n",
       "      <td>1.208494</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.449401</td>\n",
       "      <td>0.567609</td>\n",
       "      <td>-0.040427</td>\n",
       "      <td>-0.911582</td>\n",
       "      <td>0.271040</td>\n",
       "      <td>-0.410972</td>\n",
       "      <td>2.323971</td>\n",
       "      <td>0.226183</td>\n",
       "      <td>2.398908</td>\n",
       "      <td>-0.385578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-03-14 16:39:48.239563</th>\n",
       "      <td>2.366804</td>\n",
       "      <td>-1.419525</td>\n",
       "      <td>0.356341</td>\n",
       "      <td>0.679244</td>\n",
       "      <td>2.952952</td>\n",
       "      <td>0.414377</td>\n",
       "      <td>0.889363</td>\n",
       "      <td>2.163380</td>\n",
       "      <td>0.454700</td>\n",
       "      <td>1.256352</td>\n",
       "      <td>...</td>\n",
       "      <td>2.320637</td>\n",
       "      <td>0.228787</td>\n",
       "      <td>1.007314</td>\n",
       "      <td>-1.230794</td>\n",
       "      <td>0.694026</td>\n",
       "      <td>1.360371</td>\n",
       "      <td>-0.970620</td>\n",
       "      <td>1.304913</td>\n",
       "      <td>-1.543305</td>\n",
       "      <td>-0.018166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1980-03-17 16:39:48.239563</th>\n",
       "      <td>1.104342</td>\n",
       "      <td>-0.693122</td>\n",
       "      <td>-0.555143</td>\n",
       "      <td>-1.283740</td>\n",
       "      <td>0.570558</td>\n",
       "      <td>-0.309020</td>\n",
       "      <td>-2.019225</td>\n",
       "      <td>-0.153222</td>\n",
       "      <td>0.867766</td>\n",
       "      <td>-0.339598</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.428816</td>\n",
       "      <td>0.200491</td>\n",
       "      <td>-1.533220</td>\n",
       "      <td>-0.077449</td>\n",
       "      <td>1.194174</td>\n",
       "      <td>-0.406989</td>\n",
       "      <td>-1.561206</td>\n",
       "      <td>-0.957596</td>\n",
       "      <td>-0.192124</td>\n",
       "      <td>-2.333219</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                PC_1      PC_2      PC_3      PC_4      PC_5  \\\n",
       "1980-03-11 16:39:48.239563 -2.247346  3.289565 -1.367301 -2.863059 -1.412666   \n",
       "1980-03-12 16:39:48.239563 -0.956982 -0.370077 -0.070016 -3.012735  0.317126   \n",
       "1980-03-13 16:39:48.239563 -4.432080  2.690585  1.475541 -1.157080  0.586217   \n",
       "1980-03-14 16:39:48.239563  2.366804 -1.419525  0.356341  0.679244  2.952952   \n",
       "1980-03-17 16:39:48.239563  1.104342 -0.693122 -0.555143 -1.283740  0.570558   \n",
       "\n",
       "                                PC_6      PC_7      PC_8      PC_9     PC_10  \\\n",
       "1980-03-11 16:39:48.239563 -0.799908  0.213012  0.331433  0.866300  0.032951   \n",
       "1980-03-12 16:39:48.239563  0.332048 -0.771460 -0.385234  0.124513  0.067744   \n",
       "1980-03-13 16:39:48.239563  0.561810  0.509762 -1.904502  1.081916  1.208494   \n",
       "1980-03-14 16:39:48.239563  0.414377  0.889363  2.163380  0.454700  1.256352   \n",
       "1980-03-17 16:39:48.239563 -0.309020 -2.019225 -0.153222  0.867766 -0.339598   \n",
       "\n",
       "                              ...        PC_19     PC_20     PC_21     PC_22  \\\n",
       "1980-03-11 16:39:48.239563    ...     1.944196 -0.481867  0.060209 -0.277116   \n",
       "1980-03-12 16:39:48.239563    ...    -0.769808 -1.209847 -0.564018 -0.825132   \n",
       "1980-03-13 16:39:48.239563    ...    -0.449401  0.567609 -0.040427 -0.911582   \n",
       "1980-03-14 16:39:48.239563    ...     2.320637  0.228787  1.007314 -1.230794   \n",
       "1980-03-17 16:39:48.239563    ...    -2.428816  0.200491 -1.533220 -0.077449   \n",
       "\n",
       "                               PC_23     PC_24     PC_25     PC_26     PC_27  \\\n",
       "1980-03-11 16:39:48.239563 -0.534618  0.986281 -0.384909 -0.801349  0.828607   \n",
       "1980-03-12 16:39:48.239563  0.753131 -0.379044 -0.456504  1.176804 -0.593204   \n",
       "1980-03-13 16:39:48.239563  0.271040 -0.410972  2.323971  0.226183  2.398908   \n",
       "1980-03-14 16:39:48.239563  0.694026  1.360371 -0.970620  1.304913 -1.543305   \n",
       "1980-03-17 16:39:48.239563  1.194174 -0.406989 -1.561206 -0.957596 -0.192124   \n",
       "\n",
       "                               PC_28  \n",
       "1980-03-11 16:39:48.239563 -1.121230  \n",
       "1980-03-12 16:39:48.239563  0.354478  \n",
       "1980-03-13 16:39:48.239563 -0.385578  \n",
       "1980-03-14 16:39:48.239563 -0.018166  \n",
       "1980-03-17 16:39:48.239563 -2.333219  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfP.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import log_loss, accuracy_score\n",
    "\n",
    "from finance_ml.model_selection import PurgedKFold\n",
    "from finance_ml.model_selection import cv_score\n",
    "\n",
    "\n",
    "def feat_imp_MDI(forest, feat_names):\n",
    "    imp_dict = {i:tree.feature_importances_ for i, tree in enumerate(forest.estimators_)}\n",
    "    imp_df = pd.DataFrame.from_dict(imp_dict, orient='index')\n",
    "    imp_df.columns = feat_names\n",
    "    # 0 simply means not used for splitting\n",
    "    imp_df = imp_df.replace(0, np.nan)\n",
    "    imp = pd.concat({'mean': imp_df.mean(),\n",
    "                     'std': imp_df.std() * np.sqrt(imp_df.shape[0])},\n",
    "                    axis=1)\n",
    "    imp /= imp['mean'].sum()\n",
    "    return imp\n",
    "\n",
    "\n",
    "def feat_imp_MDA(clf, X, y, n_splits, sample_weight, t1, pct_embargo, scoring='neg_log_loss'):\n",
    "    if scoring not in ['neg_log_loss', 'accuracy']:\n",
    "        raise Exception('wrong scoring method')\n",
    "    cv_gen = PurgedKFold(n_splits=n_splits, t1=t1, pct_embargo=pct_embargo)\n",
    "    index = np.arange(n_splits)\n",
    "    scores = pd.Series(index=index)\n",
    "    scores_perm = pd.DataFrame(index=index, columns=X.columns)\n",
    "    for idx, (train, test) in zip(index, cv_gen.split(X=X)):\n",
    "        X_train = X.iloc[train]\n",
    "        y_train = y.iloc[train]\n",
    "        w_train = sample_weight.iloc[train]\n",
    "        X_test = X.iloc[test]\n",
    "        y_test = y.iloc[test]\n",
    "        w_test = sample_weight.iloc[test]\n",
    "        clf_fit = clf.fit(X_train, y_train, sample_weight=w_train.values)\n",
    "        if scoring == 'neg_log_loss':\n",
    "            prob = clf_fit.predict_proba(X_test)\n",
    "            scores.loc[idx] = -log_loss(y_test, prob, sample_weight=w_test.values,\n",
    "                                     labels=clf_fit.classes_)\n",
    "        else:\n",
    "            pred = clf_fit.predict(X_test)\n",
    "            scores.loc[idx] = accuracy_score(y_test,  pred, sample_weight=w_test.values)\n",
    "        \n",
    "        for col in X.columns:\n",
    "            X_test_ = X_test.copy(deep=True)\n",
    "            # Randomize certain feature to make it not effective\n",
    "            np.random.shuffle(X_test_[col].values)\n",
    "            if scoring == 'neg_log_loss':\n",
    "                prob = clf_fit.predict_proba(X_test_)\n",
    "                scores_perm.loc[idx, col] = -log_loss(y_test, prob, sample_weight=w_test.value,\n",
    "                                                    labels=clf_fit.classes_)\n",
    "            else:\n",
    "                pred = clf_fit.predict(X_test_)\n",
    "                scores_perm.loc[idx, col] = accuracy_score(y_test, pred, sample_weight=w_test.values)\n",
    "    # (Original score) - (premutated score)\n",
    "    imprv = (-scores_perm).add(scores, axis=0)\n",
    "    # Relative to maximum improvement\n",
    "    if scoring == 'neg_log_loss':\n",
    "        max_imprv = -scores_perm\n",
    "    else:\n",
    "        max_imprv = 1. - scores_perm\n",
    "    imp = imprv / max_imprv\n",
    "    imp = pd.DataFrame({'mean': imp.mean(), 'std': imp.std() * np.sqrt(imp.shape[0])})\n",
    "    return imp, scores.mean()\n",
    "    \n",
    "\n",
    "def aux_feat_imp_SFI(feat_names, clf, X, cont, scoring, cv_gen):\n",
    "    imp = pd.DataFrame(columns=['mean', 'std'])\n",
    "    for feat_name in feat_names:\n",
    "        scores = cv_score(clf, X=X[[feat_name]], y=cont['bin'],\n",
    "                          sample_weight=cont['w'],\n",
    "                          scoring=scoring,\n",
    "                          cv_gen=cv_gen)\n",
    "        imp.loc[feat_name, 'mean'] = scores.mean()\n",
    "        imp.loc[feat_name, 'std'] = scores.std() * np.sqrt(scores.shape[0])\n",
    "    return imp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "\n",
    "from finance_ml.multiprocessing import mp_pandas_obj\n",
    "from finance_ml.model_selection import PurgedKFold\n",
    "\n",
    "\n",
    "def feat_importance(X, cont, clf=None, n_estimators=1000, n_splits=10, max_samples=1.,\n",
    "                    num_threads=24, pct_embargo=0., scoring='accuracy',\n",
    "                    method='SFI', min_w_leaf=0., **kwargs):\n",
    "    n_jobs = (-1 if num_threads > 1 else 1)\n",
    "    # Build classifiers\n",
    "    if clf is None:\n",
    "        base_clf = DecisionTreeClassifier(criterion='entropy', max_features=1,\n",
    "                                          class_weight='balanced',\n",
    "                                          min_weight_fraction_leaf=min_w_leaf)\n",
    "        clf = BaggingClassifier(base_estimator=base_clf, n_estimators=n_estimators,\n",
    "                                max_features=1., max_samples=max_samples,\n",
    "                                oob_score=True, n_jobs=n_jobs)\n",
    "    fit_clf = clf.fit(X, cont['bin'], sample_weight=cont['w'].values)\n",
    "    if hasattr(fit_clf, 'oob_score_'):\n",
    "        oob = fit_clf.oob_score_\n",
    "    else:\n",
    "        oob = None\n",
    "    if method == 'MDI':\n",
    "        imp = feat_imp_MDI(fit_clf, feat_names=X.columns)\n",
    "        oos = cv_score(clf, X=X, y=cont['bin'], n_splits=n_splits,\n",
    "                       sample_weight=cont['w'], t1=cont['t1'],\n",
    "                       pct_embargo=pct_embargo, scoring=scoring).mean()\n",
    "    elif method == 'MDA':\n",
    "        imp, oos = feat_imp_MDA(clf, X=X, y=cont['bin'], n_splits=n_splits,\n",
    "                                sample_weight=cont['w'], t1=cont['t1'],\n",
    "                                pct_embargo=pct_embargo, scoring=scoring)\n",
    "    elif method == 'SFI':\n",
    "        cv_gen = PurgedKFold(n_splits=n_splits, t1=cont['t1'], pct_embargo=pct_embargo)\n",
    "        oos = cv_score(clf, X=X, y=cont['bin'], sample_weight=cont['w'],\n",
    "                       scoring=scoring, cv_gen=cv_gen)\n",
    "        clf.n_jobs = 1\n",
    "        imp = mp_pandas_obj(aux_feat_imp_SFI, ('feat_names', X.columns),\n",
    "                            num_threads, clf=clf, X=X, cont=cont,\n",
    "                            scoring=scoring, cv_gen=cv_gen)\n",
    "    return imp, oob, oos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          mean       std\n",
      "PC_1  0.105600  0.161993\n",
      "PC_2  0.087906  0.162054\n",
      "PC_3  0.072493  0.111387\n",
      "PC_4  0.105228  0.159358\n",
      "PC_5  0.111355  0.159081\n",
      "0.8682\n",
      "0.7874999999999999\n",
      "CPU times: user 32.1 s, sys: 0 ns, total: 32.1 s\n",
      "Wall time: 32.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "clf = RandomForestClassifier(oob_score=True, n_estimators=100) \n",
    "imp_MDI, oob_MDI, oos_MDI = feat_importance(dfP, cont, clf=clf, method='MDI')\n",
    "print(imp_MDI.head())\n",
    "print(oob_MDI)\n",
    "print(oos_MDI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          mean       std\n",
      "PC_1  0.176057  0.433173\n",
      "PC_2  0.212094  0.372648\n",
      "PC_3  0.195688  0.543541\n",
      "PC_4  0.089051  0.372906\n",
      "PC_5  0.223289  0.429095\n",
      "0.8718\n",
      "0.7842\n",
      "CPU times: user 37.6 s, sys: 0 ns, total: 37.6 s\n",
      "Wall time: 37.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "clf = RandomForestClassifier(oob_score=True, n_estimators=100) \n",
    "imp_MDA, oob_MDA, oos_MDA = feat_importance(dfP, cont, clf=clf, method='MDA')\n",
    "print(imp_MDA.head())\n",
    "print(oob_MDA)\n",
    "print(oos_MDA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-07-09 16:47:41.320736 95.83% aux_feat_imp_SFI done after 3.05 minutes. Remaining 0.13 minutes.\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         mean        std\n",
      "PC_1   0.4977   0.163945\n",
      "PC_10  0.4561  0.0643032\n",
      "PC_11  0.4536  0.0870655\n",
      "PC_12  0.4516  0.0658058\n",
      "PC_13  0.4513  0.0707821\n",
      "0.8682\n",
      "[0.749 0.712 0.865 0.718 0.708 0.79  0.755 0.875 0.813 0.859]\n",
      "CPU times: user 35.6 s, sys: 784 ms, total: 36.4 s\n",
      "Wall time: 3min 39s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-07-09 16:47:43.442189 100.0% aux_feat_imp_SFI done after 3.08 minutes. Remaining 0.0 minutes.\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "clf = RandomForestClassifier(oob_score=True, n_estimators=100) \n",
    "imp_SFI, oob_SFI, oos_SFI = feat_importance(dfP, cont, clf=clf, method='SFI')\n",
    "print(imp_SFI.head())\n",
    "print(oob_SFI)\n",
    "print(oos_SFI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['PC_5', 'PC_1', 'PC_4', 'PC_3', 'PC_2', 'PC_23', 'PC_16', 'PC_27',\n",
       "       'PC_22', 'PC_15', 'PC_21', 'PC_14', 'PC_7', 'PC_18', 'PC_28', 'PC_6',\n",
       "       'PC_26', 'PC_10', 'PC_9', 'PC_11', 'PC_24', 'PC_25', 'PC_12', 'PC_13',\n",
       "       'PC_20', 'PC_17', 'PC_8', 'PC_19'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp_SFI.sort_values('mean', ascending=False).index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['PC_5', 'PC_1', 'PC_4', 'PC_2', 'PC_3', 'PC_28', 'PC_6', 'PC_27',\n",
       "       'PC_26', 'PC_22', 'PC_25', 'PC_17', 'PC_14', 'PC_7', 'PC_23', 'PC_21',\n",
       "       'PC_19', 'PC_13', 'PC_16', 'PC_20', 'PC_24', 'PC_8', 'PC_12', 'PC_9',\n",
       "       'PC_15', 'PC_18', 'PC_11', 'PC_10'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp_MDI.sort_values('mean', ascending=False).index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['PC_5', 'PC_2', 'PC_3', 'PC_1', 'PC_6', 'PC_28', 'PC_4', 'PC_27',\n",
       "       'PC_26', 'PC_23', 'PC_15', 'PC_21', 'PC_20', 'PC_22', 'PC_11', 'PC_17',\n",
       "       'PC_14', 'PC_12', 'PC_10', 'PC_13', 'PC_7', 'PC_18', 'PC_8', 'PC_16',\n",
       "       'PC_9', 'PC_19', 'PC_24', 'PC_25'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp_MDA.sort_values('mean', ascending=False).index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "They agree on each other somewhat because PCA reduces duplication of features. Only SFI is not impacted by substitution effects."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tilde = pd.concat((X, dfP), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 68)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tilde.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         mean       std\n",
      "I_0  0.038626  0.190660\n",
      "I_1  0.016350  0.080696\n",
      "I_2  0.031043  0.161967\n",
      "I_3  0.024104  0.127805\n",
      "I_4  0.034054  0.206238\n",
      "0.9443\n",
      "0.917\n",
      "CPU times: user 54 s, sys: 0 ns, total: 54 s\n",
      "Wall time: 54 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "clf = RandomForestClassifier(oob_score=True, n_estimators=100) \n",
    "imp_MDI, oob_MDI, oos_MDI = feat_importance(X_tilde, cont, clf=clf, method='MDI')\n",
    "print(imp_MDI.head())\n",
    "print(oob_MDI)\n",
    "print(oos_MDI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         mean       std\n",
      "I_0  0.121837  0.332719\n",
      "I_1  0.025352  0.214111\n",
      "I_2  0.014875  0.273832\n",
      "I_3  0.056271  0.156276\n",
      "I_4  0.054778  0.360993\n",
      "0.9462\n",
      "0.9149999999999998\n",
      "CPU times: user 1min 4s, sys: 0 ns, total: 1min 4s\n",
      "Wall time: 1min 4s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "clf = RandomForestClassifier(oob_score=True, n_estimators=100) \n",
    "imp_MDA, oob_MDA, oos_MDA = feat_importance(X_tilde, cont, clf=clf, method='MDA')\n",
    "print(imp_MDA.head())\n",
    "print(oob_MDA)\n",
    "print(oos_MDA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-07-09 17:06:30.105358 100.0% aux_feat_imp_SFI done after 6.5 minutes. Remaining 0.0 minutes.s..\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       mean        std\n",
      "I_0  0.4789   0.161984\n",
      "I_1   0.458  0.0713162\n",
      "I_2   0.437  0.0538331\n",
      "I_3  0.4591   0.103339\n",
      "I_4  0.4934   0.168001\n",
      "0.945\n",
      "[0.916 0.91  0.954 0.917 0.913 0.913 0.876 0.93  0.917 0.918]\n",
      "CPU times: user 52.1 s, sys: 1.11 s, total: 53.3 s\n",
      "Wall time: 7min 21s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "clf = RandomForestClassifier(oob_score=True, n_estimators=100) \n",
    "imp_SFI, oob_SFI, oos_SFI = feat_importance(X_tilde, cont, clf=clf, method='SFI')\n",
    "print(imp_SFI.head())\n",
    "print(oob_SFI)\n",
    "print(oos_SFI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MDI\n",
      "Index(['R_1', 'R_9', 'PC_4', 'I_8', 'I_0', 'PC_5', 'R_4', 'R_3', 'I_4', 'R_6',\n",
      "       'I_7', 'I_2', 'I_9', 'PC_1', 'I_6', 'I_5', 'R_7', 'PC_2', 'I_3',\n",
      "       'PC_28', 'PC_6', 'R_2', 'PC_3', 'R_8', 'R_0', 'I_1', 'R_5', 'PC_27',\n",
      "       'PC_26', 'PC_17', 'PC_25', 'PC_12', 'PC_22', 'PC_9', 'N_17', 'N_9',\n",
      "       'PC_21', 'N_16', 'N_6', 'PC_18', 'PC_15', 'N_14', 'N_7', 'N_0', 'PC_19',\n",
      "       'PC_20', 'PC_24', 'PC_8', 'PC_13', 'PC_14', 'N_19', 'N_18', 'PC_7',\n",
      "       'N_13', 'N_8', 'PC_11', 'N_5', 'N_12', 'N_3', 'N_15', 'PC_10', 'N_1',\n",
      "       'N_10', 'PC_23', 'N_2', 'N_11', 'PC_16', 'N_4'],\n",
      "      dtype='object')\n",
      "MDA\n",
      "Index(['I_8', 'I_0', 'I_7', 'R_9', 'I_6', 'R_3', 'I_3', 'PC_6', 'I_4', 'I_9',\n",
      "       'R_4', 'PC_28', 'R_8', 'I_1', 'R_1', 'R_2', 'I_2', 'PC_27', 'N_14',\n",
      "       'I_5', 'N_9', 'PC_24', 'PC_4', 'PC_11', 'N_2', 'N_0', 'PC_9', 'N_3',\n",
      "       'PC_12', 'N_12', 'PC_13', 'PC_14', 'PC_10', 'PC_19', 'PC_15', 'N_10',\n",
      "       'PC_7', 'PC_21', 'PC_23', 'N_6', 'PC_18', 'PC_3', 'R_6', 'PC_25',\n",
      "       'N_19', 'N_5', 'R_0', 'PC_8', 'N_8', 'N_11', 'PC_16', 'PC_22', 'PC_5',\n",
      "       'N_13', 'N_4', 'R_7', 'N_17', 'N_16', 'N_15', 'N_1', 'PC_20', 'N_18',\n",
      "       'PC_17', 'PC_26', 'N_7', 'R_5', 'PC_1', 'PC_2'],\n",
      "      dtype='object')\n",
      "SFI\n",
      "Index(['R_1', 'PC_5', 'R_3', 'PC_1', 'I_4', 'PC_4', 'R_7', 'I_5', 'I_0', 'R_6',\n",
      "       'R_4', 'PC_3', 'I_8', 'I_7', 'R_0', 'PC_2', 'R_8', 'R_9', 'R_2',\n",
      "       'PC_23', 'N_0', 'N_13', 'PC_16', 'N_8', 'N_1', 'PC_27', 'PC_22',\n",
      "       'PC_15', 'N_3', 'PC_21', 'PC_14', 'R_5', 'PC_7', 'I_3', 'PC_18',\n",
      "       'PC_28', 'I_1', 'N_19', 'N_11', 'PC_6', 'N_14', 'N_5', 'N_6', 'PC_26',\n",
      "       'PC_10', 'N_15', 'PC_9', 'N_10', 'PC_24', 'PC_25', 'N_12', 'PC_11',\n",
      "       'PC_12', 'PC_13', 'N_9', 'N_18', 'N_4', 'N_16', 'PC_20', 'N_2', 'N_17',\n",
      "       'PC_17', 'I_9', 'PC_8', 'N_7', 'PC_19', 'I_2', 'I_6'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(\"MDI\")\n",
    "print(imp_MDI.sort_values('mean', ascending=False).index)\n",
    "print(\"MDA\")\n",
    "print(imp_MDA.sort_values('mean', ascending=False).index)\n",
    "print(\"SFI\")\n",
    "print(imp_SFI.sort_values('mean', ascending=False).index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "They do not agree on each other because of substitution effects."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8.3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MDI and MDA changes the columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
